Connecting... Still working... 

df = spark.read.format("csv").option("header", "true").load("dbfs:/FileStore/shared_uploads/smoturu@gmu.edu/Adult_final.csv")
display(df)

#display the summary
display(df.summary())

from functools import reduce

oldColumns = df.schema.names
newColumns = [
     "age",
     "workclass",
     "finalweight",
     "education",
     "maritalstatus",
     "occupation",
     "relationship",
     "race",
     "sex",
     "capitalgain",
     "capitalloss",
     "hoursperweek",
     "nativecountry",
     "income"
 ]

df = reduce(lambda df, idx: df.withColumnRenamed(oldColumns[idx], newColumns[idx]), range(len(oldColumns)), df)
df.printSchema()

df.display()

from pyspark.sql.functions import col

# Assuming your DataFrame is called "df"
df = df.withColumn("hoursperweek", col("hoursperweek").cast("integer"))
df = df.withColumn("age", col("age").cast("integer"))
df = df.withColumn("finalweight", col("finalweight").cast("integer"))
df = df.withColumn("capitalgain", col("capitalgain").cast("integer"))
df = df.withColumn("capitalloss", col("capitalloss").cast("integer"))

df.display()

df.printSchema()

#Encoding the data for GENDER where 0 = Female, 1 = Male
from pyspark.sql.functions import when
 
df = df.withColumn("income", when(df["income"] == "<=50K", 0)
                                            .otherwise(1))
 
df.select("income").distinct().show()

df.display()

from pyspark.sql.functions import col

new_df = df.filter(col('workclass') == 'Private')
print(new_df.filter(col('income') == 1).count())
new_df.filter(col('income') == 1).show(2)

#Checking the ratio of male and female according to their income
import pyspark.sql.functions as F

class_sex = df.crosstab('income', 'sex')
class_sex.display()

# Import required libraries
from pyspark.ml.feature import VectorAssembler
from pyspark.ml.stat import Correlation
import seaborn as sns
import matplotlib.pyplot as plt

# Select only numeric columns
numeric_cols = [c for c,t in df.dtypes if t in ['double', 'int']]

# Create vector assembler to combine numeric columns into a single vector column
assembler = VectorAssembler(inputCols=numeric_cols, outputCol='features')
assembled_df = assembler.transform(df).select('features')

# Compute correlation matrix
correlation_matrix = Correlation.corr(assembled_df, 'features').head()

# Convert correlation matrix to a Pandas DataFrame for plotting
corr_matrix_pd = correlation_matrix[0].toArray()
corr_matrix = pd.DataFrame(corr_matrix_pd, columns=numeric_cols, index=numeric_cols)

# Plot correlation matrix
fig, ax = plt.subplots(figsize=(10,8))
sns.heatmap(data=corr_matrix, annot=True, cmap='viridis')
ax.set_title('Adult Census Correlation Matrix', fontsize=15);

# filter the DataFrame to only include those with income equal to 1
df_filtered = df.filter(df.income == 1)

# convert the PySpark DataFrame to a Pandas DataFrame
pandas_df = df_filtered.select('age').toPandas()

# create a histogram of age using seaborn
sns.histplot(data=pandas_df, x='age', bins=20)
plt.title('Age Distribution for Income > 50K')
plt.xlabel('Age')
plt.ylabel('Frequency')
plt.show()

# filter the DataFrame to only include those with income equal to 1
df_filtered = df.filter(df.income == 1)

# group the filtered DataFrame by work class and count the number of instances
grouped_df = df_filtered.groupBy('workclass').count()

# convert the PySpark DataFrame to a Pandas DataFrame
pandas_df = grouped_df.toPandas()

# create a pie chart of work class using matplotlib
plt.figure(figsize=(6, 6))
plt.pie(pandas_df['count'], labels=pandas_df['workclass'], autopct='%1.1f%%', textprops={'fontsize': 8})

plt.title('Work Class Distribution for Income > 50K', fontsize=14)
plt.show()

# Creating a pandas dataframe
pdf = df.select(col("age"), col("hoursperweek")).toPandas()

# Plotting a scatter plot using Seaborn
sns.scatterplot(x="age", y="hoursperweek", data=pdf)
plt.xlabel('Age')
plt.ylabel('Hours per Week')
plt.title('Age vs. Hours per Week')
plt.show()

import plotly.express as px

fig = px.scatter(df.toPandas(), x="age", y="hoursperweek", color="income", hover_data=['education', 'occupation'])
fig.update_layout(title='Age vs. Hours per Week', xaxis_title='Age', yaxis_title='Hours per Week')
fig.show()

import plotly.express as px

fig = px.histogram(df.toPandas(), x="age", color="income", nbins=30, 
                   marginal="box", hover_data=['education', 'occupation'])
fig.update_layout(title='Distribution of Age by Income', xaxis_title='Age')
fig.show()

import plotly.express as px

# Define custom colors for the histogram bars
colors = {'<=50K': 'rgb(31, 100, 180)', '>50K': 'rgb(255, 117, 14)'}

fig = px.histogram(df.toPandas(), x="age", color="income", nbins=20, 
                   marginal="box", hover_data=['education', 'occupation'],
                   color_discrete_map=colors)
fig.update_layout(title='Distribution of Age by Income', xaxis_title='Age')
fig.show()

import plotly.express as px

fig = px.violin(df.toPandas(), y="age", x="income", box=True, points="all",
                hover_data=['education', 'occupation'])
fig.update_layout(title='Distribution of Age by Income', yaxis_title='Age')
fig.show()